/**
 * @author Fang Lu, fanglupku@gmail.com
 * 2012-2-19
 */
package com.autotag.data.extract;

import java.io.File;
import java.sql.Connection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.autotag.dao.ServiceDao;
import com.autotag.dao.ServiceTagDao;
import com.autotag.dao.TagDao;
import com.autotag.data.crawler.ServiceDetailCrawler;
import com.autotag.entity.Service;
import com.autotag.entity.Tag;

import fanglu.util.FileDealer;
import fanglu.util.SQLDealer;

public class ServiceDetailExtractor {

	private static Pattern WSDLPattern = Pattern
			.compile(
					"href=\"http://webservices.seekda.com/cache(.*?)type=xml\">",
					Pattern.CASE_INSENSITIVE|Pattern.DOTALL);
	private static Pattern tagItemPattern = Pattern.compile(
			"<a.*?rel=\"tag\".*?>(.+?)</a>",
			Pattern.CASE_INSENSITIVE);

	public static String extractWSDLFromServiceDetailFile(String pageContent) {
		String result = "";
		Matcher matcher = WSDLPattern.matcher(pageContent);
		if (matcher.find()) {
			result = "http://webservices.seekda.com/cache"
					+ matcher.group(1) + "type=xml";
		}
		return result;
	}

	public static List<Tag> extractTagListFromServiceDetailFile(
			String pageContent) {
		ArrayList<Tag> resultList = new ArrayList<Tag>();
		Matcher matcher = tagItemPattern.matcher(pageContent);
		while (matcher.find()) {
			String thisString = new String(pageContent.substring(matcher.start(), matcher.end()));
			int hrefStart = thisString.indexOf("href=\"");
			String ref = "";
			if (hrefStart > 0){
				hrefStart += "href=\"".length();
				int hrefEnd = thisString.indexOf("\"", hrefStart);
				ref = new String(thisString.substring(hrefStart, hrefEnd));
			}
			String name = matcher.group(1);
			Tag thisTag = new Tag(-1, name, ref);
			resultList.add(thisTag);
		}
		return resultList;
	}

	public static boolean updateServiceDetailFromServiceDetailFile(
			Service originService, String pageContent) {
		try {
			originService
					.setWSDL(extractWSDLFromServiceDetailFile(pageContent));
//			System.out.println("WSDL:" + originService.getWSDL());
			originService
					.setTagList(extractTagListFromServiceDetailFile(pageContent));
			System.out.println("tag list size: " + originService.getTagList().size());
			return true;
		} catch (Exception e) {
			e.printStackTrace();
			return false;
		}
	}

	public static void processAllServiceDetailFile(String baseDir,
			String encoding, String dbURL, String user, String pwd) {
		Connection conn = SQLDealer.getDbConnection(dbURL, user, pwd);
		List<Service> sList = ServiceDao.getAllService(conn);
		Iterator<Service> itor = sList.iterator();
		HashMap<Tag, ArrayList<Service>> tagMapServiceList = new HashMap<Tag, ArrayList<Service>>();
		int serviceIndex = 0;
		while (itor.hasNext()) {
			Service thisService = itor.next();
			++serviceIndex;
			thisService.setProviderName(ServiceDetailCrawler
					.extractProviderFromRef(thisService.getRefSeekda()));
			thisService.setName(ServiceDetailCrawler.extractServiceNameFromRef(thisService.getRefSeekda(), thisService.getProviderName()));
			String thisProviderLowerName = thisService.getProviderName()
					.toLowerCase();
			String subDirName = "/other";
			if (thisProviderLowerName.toLowerCase().charAt(0) >= 'a'
					&& thisProviderLowerName.toLowerCase().charAt(0) <= 'z') {
				subDirName = "/"
						+ thisProviderLowerName.toLowerCase().charAt(0);
			}
			String thisDirName = baseDir + subDirName + "/"
					+ thisService.getProviderName();
			String serviceFileName = thisDirName
					+ "/"
					+ thisService.getName().replace('?', '.').replace('/', '.')
							.replace('-', '.') + ".htm";
			File thisFile = new File(serviceFileName);
			int retryNum = 0;
			while (!thisFile.exists() && retryNum < 3) {
				System.out.println((serviceIndex) + " / " + sList.size() + " is re-crawled!");
				System.out.println(serviceFileName);
				if (ServiceDetailCrawler.fetchService(thisService, baseDir,
						encoding, true)) {
					break;
				}
				++retryNum;
			}
			if (thisFile.exists()) {
				try {
					String pageContent = FileDealer.getFileContent(
							serviceFileName, encoding);
					updateServiceDetailFromServiceDetailFile(thisService,
							pageContent);
					ServiceDao.updateService(conn, thisService);
					Iterator<Tag> tag_itor = thisService.getTagList()
							.iterator();
					while (tag_itor.hasNext()) {
						Tag thisTag = tag_itor.next();
						if (tagMapServiceList.containsKey(thisTag)) {
							ArrayList<Service> thisList = tagMapServiceList
									.get(thisTag);
							thisList.add(thisService);
						} else {
							ArrayList<Service> thisList = new ArrayList<Service>();
							thisList.add(thisService);
							tagMapServiceList.put(thisTag, thisList);
						}
					}
				} catch (Exception e) {
					e.printStackTrace();
					System.out.println("update error: id="
							+ thisService.getId());
				}
			}
			System.out.println((serviceIndex) + " / " + sList.size() + " is OK! step 1!");
		}
		Set<Tag> keySet = tagMapServiceList.keySet();
		Iterator<Tag> tag_itor = keySet.iterator();
		int tagIndex = 0;
		while (tag_itor.hasNext()) {
			++ tagIndex;
			Tag thisTag = tag_itor.next();
			TagDao.insertTag(conn, thisTag);
			System.out.println((tagIndex) + " / " + keySet.size() + " is OK! step 2!");
		}
		
		List<Tag> allTagList = TagDao.getAllTag(conn);
		tag_itor = allTagList.iterator();
		tagIndex = 0;
		while (tag_itor.hasNext()) {
			++ tagIndex;
			Tag thisTag = tag_itor.next();
			if (tagMapServiceList.containsKey(thisTag)) {
				ArrayList<Service> thisList = tagMapServiceList.get(thisTag);
				for (int i = 0; i < thisList.size(); ++i) {
					Service thisService = thisList.get(i);
					ServiceTagDao.insertServiceTag(conn, thisService, thisTag);
				}
			}
			System.out.println((tagIndex) + " / " + allTagList.size() + " is OK! step 3!");
		}
	}
	public static void main(String args[]){
		processAllServiceDetailFile("d:\\seekda_data\\service", "utf8",
				"jdbc:mysql://localhost:3306/autotag", "root", "ftc86524");
	}
}
